{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a98e6dac-e368-4310-a0cb-751faf1e3b9b",
   "metadata": {},
   "source": [
    "The following notebook example will allow you to test DPK, without cloning the repo. You can run it either on Google Colab or you can use your local environment (by downloading just the notebook). We use a temporary folder for input and output, but users are encouraged to use their own input folder. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4f9c952-cb3b-40f1-bfb5-00d9a43a5715",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "%pip install 'data-prep-toolkit-transforms[docling2parquet]==1.1.1'\n",
    "%pip install pandas\n",
    "import urllib.request\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d447b17",
   "metadata": {},
   "outputs": [],
   "source": [
    "shutil.os.makedirs(\"tmp/input\", exist_ok=True)\n",
    "urllib.request.urlretrieve(\"https://raw.githubusercontent.com/data-prep-kit/data-prep-kit/dev/transforms/language/docling2parquet/test-data/input/archive1.zip\", \"tmp/input/archive1.zip\")\n",
    "urllib.request.urlretrieve(\"https://raw.githubusercontent.com/data-prep-kit/data-prep-kit/dev/transforms/language/docling2parquet/test-data/input/redp5110-ch1.pdf\", \"tmp/input/redp5110-ch1.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76469c87-cfd0-4acd-b57f-36edc52018ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dpk_docling2parquet import Docling2Parquet\n",
    "from dpk_docling2parquet import docling2parquet_contents_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90277e74-f223-434d-b256-94079bd6c5bc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Docling2Parquet(input_folder= \"tmp/input\", \n",
    "               output_folder= \"tmp/output\", \n",
    "               data_files_to_use=['.pdf', '.zip'],\n",
    "               docling2parquet_contents_type=docling2parquet_contents_types.JSON).transform()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2bee8da-c566-4e45-bca1-354dfd04b0df",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyarrow.parquet as pq\n",
    "import pandas as pd\n",
    "table = pq.read_table('tmp/output/archive1.parquet')\n",
    "table.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "685229ce-698b-4a97-91f8-b8533337bc0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "table = pq.read_table('tmp/output/redp5110-ch1.parquet')\n",
    "table.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "deeef367-5846-4429-bc22-c395fd7548bc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
